DRYed up website_agent code a bit
Albert Sun
1 changed files with 18 additions and 7 deletions
+ 18
- 7
app/models/agents/website_agent.rb
@@ -62,7 +62,7 @@ module Agents |
||
| 62 | 62 |
|
| 63 | 63 |
def validate_options |
| 64 | 64 |
errors.add(:base, "url and expected_update_period_in_days are required") unless options[:expected_update_period_in_days].present? && options[:url].present? |
| 65 |
- if !options[:extract].present? && options[:type] != "json" |
|
| 65 |
+ if !options[:extract].present? && extraction_type != "json" |
|
| 66 | 66 |
errors.add(:base, "extract is required for all types except json") |
| 67 | 67 |
end |
| 68 | 68 |
end |
@@ -76,16 +76,15 @@ module Agents |
||
| 76 | 76 |
end |
| 77 | 77 |
request.on_success do |response| |
| 78 | 78 |
doc = parse(response.body) |
| 79 |
- output = {}
|
|
| 80 |
- previous_payloads = events.order("id desc").limit(UNIQUENESS_LOOK_BACK).pluck(:payload).map(&:to_json) if options[:mode].to_s == "on_change"
|
|
| 81 | 79 |
|
| 82 |
- if extraction_type == "json" && !options[:extract].present? |
|
| 80 |
+ if extract_full_json? |
|
| 83 | 81 |
result = doc |
| 84 |
- if !options[:mode] || options[:mode].to_s == "all" || (options[:mode].to_s == "on_change" && !previous_payloads.include?(result.to_json)) |
|
| 82 |
+ if store_payload? result |
|
| 85 | 83 |
log "Storing new result for '#{name}': #{result.inspect}"
|
| 86 | 84 |
create_event :payload => result |
| 87 | 85 |
end |
| 88 | 86 |
else |
| 87 |
+ output = {}
|
|
| 89 | 88 |
options[:extract].each do |name, extraction_details| |
| 90 | 89 |
result = if extraction_type == "json" |
| 91 | 90 |
output[name] = Utils.values_at(doc, extraction_details[:path]) |
@@ -120,8 +119,8 @@ module Agents |
||
| 120 | 119 |
end |
| 121 | 120 |
end |
| 122 | 121 |
|
| 123 |
- if !options[:mode] || options[:mode].to_s == "all" || (options[:mode].to_s == "on_change" && !previous_payloads.include?(result.to_json)) |
|
| 124 |
- log "Storing new result for '#{name}': #{result.inspect}"
|
|
| 122 |
+ if store_payload? result |
|
| 123 |
+ log "Storing new parsed result for '#{name}': #{result.inspect}"
|
|
| 125 | 124 |
create_event :payload => result |
| 126 | 125 |
end |
| 127 | 126 |
end |
@@ -133,6 +132,18 @@ module Agents |
||
| 133 | 132 |
|
| 134 | 133 |
private |
| 135 | 134 |
|
| 135 |
+ def store_payload? result |
|
| 136 |
+ !options[:mode] || options[:mode].to_s == "all" || (options[:mode].to_s == "on_change" && !previous_payloads.include?(result.to_json)) |
|
| 137 |
+ end |
|
| 138 |
+ |
|
| 139 |
+ def previous_payloads |
|
| 140 |
+ events.order("id desc").limit(UNIQUENESS_LOOK_BACK).pluck(:payload).map(&:to_json) if options[:mode].to_s == "on_change"
|
|
| 141 |
+ end |
|
| 142 |
+ |
|
| 143 |
+ def extract_full_json? |
|
| 144 |
+ (!options[:extract].present? && extraction_type == "json") |
|
| 145 |
+ end |
|
| 146 |
+ |
|
| 136 | 147 |
def extraction_type |
| 137 | 148 |
(options[:type] || begin |
| 138 | 149 |
if options[:url] =~ /\.(rss|xml)$/i |